import json

from re_common.baselibrary.tools.all_requests.mrequest import MRequest
from re_common.baselibrary.utils.basedict import BaseDicts
from re_common.baselibrary.utils.core.requests_core import INSIDE_HEADERS

from apps.core.global_model import InputPlatformModel
from apps.crawler_platform.core_api.models import SelectMongoModel
from apps.crawler_platform.util.requestapihelper import RequestApiHelper
from apps.crawler_platform.util.sqlhelper import SQLHelper
from apps.sql_app.mmongodb import Coll
from settings import URLDISTRIBUTED


class GetJournalHtmlDetail(object):
    def __init__(self):
        self.b_list = list()
        self.a_list = list()
        self.a_dict = dict()
        self.issue_dict = dict()

    def check_meta(self, rule, data):
        if "page_index" in rule:
            if len(rule) - 3 == len(data):
                for i in rule:
                    if i != "task_name" and i != "task_tag" and i != "page_index":
                        meta_1 = data.get(i)
                        if meta_1 is None:
                            return False, "no_value"
                return True, ""
            else:
                return False, "not_equal"
        else:
            if len(rule) - 2 == len(data):
                for i in rule:
                    if i != "task_name" and i != "task_tag":
                        meta_1 = data.get(i)
                        if meta_1 is None:
                            return False, "no_value"
                return True, ""
            else:
                return False, "not_equal"

    def check_article_meta(self, data):
        if len(data) == 1:
            for k, v in data.items():
                if k == "lngid":
                    return True, ""
            return False, "no_value"
        else:
            return False, "not_equal"

    def check_issue_mode(self, id_list):
        if "page_index" in id_list:
            return "on"
        else:
            return "off"

    async def set_task_list(self, task_info, infos, data_dicts_list):
        task_name = task_info["task_name"]
        task_tag = task_info["task_tag"]
        # redis里面的id_list
        id_list_parse = task_info["id_list"]
        rule_list = eval(id_list_parse)
        # 对需要的元素个数进行检查, 取传进来的id列表中第一个元素进行检查, 默认后续数据与第一个格式相同, 约定以字典形式进行传入
        bools, msg = self.check_meta(rule_list, data_dicts_list[0])
        if bools:

            new_id_list = ["_".join([task_name, task_tag, BaseDicts.is_dict_exit_key(data_dict, "journal_rawid")]) for
                           data_dict in
                           data_dicts_list]
            nosql_table = task_info["nosql_table"]
            conn = Coll.get_table_conn(nosql_table)
            data_list = conn.find({"_id": {"$in": new_id_list}}, infos)
            async for data in data_list:
                local_dict = dict()
                last_data = data["last_data"]
                down_dict = last_data["down_dict"]
                local_key = data["_id"]
                local_dict[local_key] = down_dict
                self.b_list.append(local_dict)
            return True, ""
        else:
            if msg == "no_value":
                return False, "所传字段错误, 请检查传输字典键: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)
            elif msg == "not_equal":
                return False, "所传字段数量错误, 查询所需元素: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)

    async def set_task_issue(self, task_info, infos, data_dicts_list):
        task_name = task_info["task_name"]
        task_tag = task_info["task_tag"]
        # redis里面的id_list
        id_list_parse = task_info["id_list"]
        nosql_table = task_info["nosql_table"]
        rule_list = eval(id_list_parse)
        judge = self.check_issue_mode(rule_list)
        if judge == "on":
            bools, msg = self.check_meta(rule_list, data_dicts_list[0])
            if bools:
                new_id_list = list()
                for data_dict in data_dicts_list:

                    journal_rawid = data_dict["journal_rawid"]
                    pub_year = data_dict["pub_year"]
                    num = data_dict["num"]
                    _id = "_".join([task_name, task_tag, journal_rawid, pub_year, num])
                    # 这个列表用于将这个键下所有页码的id记录在内
                    self.a_dict[_id] = []
                    self.issue_dict[_id] = []
                    sql = f"select page from journal_issue where task_name = '{task_name}' and task_tag = '{task_tag}' and journal_rawid = '{journal_rawid}' and pub_year = '{pub_year}' and num = '{num}' and page_index = 1"
                    from main import app
                    pool = app.state.pool
                    data = await SQLHelper.fetchone(sql)
                    for i in range(1, int(data[0]) + 1):
                        one_data_list = [task_name, task_tag, journal_rawid, pub_year, num]
                        one_data_list.append(str(i))
                        one_page_id = "_".join(one_data_list)
                        # 这个列表用于等会统一查询
                        new_id_list.append(one_page_id)
                        self.a_dict[_id].append(one_page_id)

                conn = Coll.get_table_conn(nosql_table)
                data_list = conn.find({"_id": {"$in": new_id_list}}, infos)
                async for data in data_list:
                    local_dict = dict()
                    last_data = data["last_data"]
                    down_dict = last_data["down_dict"]
                    local_key = data["_id"]
                    local_dict[local_key] = down_dict
                    for key, value in self.a_dict.items():
                        if local_key in value:
                            self.issue_dict[key].append(local_dict)
                for k, v in self.issue_dict.items():
                    self.b_list.append({k: v})
                return True, ""
            else:
                if msg == "no_value":
                    return False, "所传字段错误, 请检查传输字典键: {}, task_name与task_tag与page_index不需要传入,其余需传入".format(id_list_parse)
                elif msg == "not_equal":
                    return False, "所传字段数量错误, 查询所需元素: {}, task_name与task_tag与page_index不需要传入,其余需传入".format(id_list_parse)

        elif judge == "off":
            # 对需要的元素个数进行检查, 取传进来的id列表中第一个元素进行检查, 默认后续数据与第一个格式相同, 约定以字典形式进行传入
            bools, msg = self.check_meta(rule_list, data_dicts_list[0])
            if bools:
                new_id_list = list()
                for data_dict in data_dicts_list:
                    one_data_list = [task_name, task_tag]
                    for list_meta in rule_list:
                        if list_meta not in ["task_name", "task_tag"]:
                            meta_data = BaseDicts.is_dict_exit_key(data_dict, list_meta)
                            one_data_list.append(meta_data)
                            data_id = "_".join(one_data_list)
                            new_id_list.append(data_id)
                conn = Coll.get_table_conn(nosql_table)
                data_list = conn.find({"_id": {"$in": new_id_list}}, infos)
                async for data in data_list:
                    local_dict = dict()
                    last_data = data["last_data"]
                    down_dict = last_data["down_dict"]
                    local_key = data["_id"]
                    local_dict[local_key] = down_dict
                    self.b_list.append(local_dict)
                return True, ""
            else:
                if msg == "no_value":
                    return False, "所传字段错误, 请检查传输字典键: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)
                elif msg == "not_equal":
                    return False, "所传字段数量错误, 查询所需元素: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)
        else:
            return False, ""

    async def set_task_article(self, task_info, infos, data_dicts_list):
        task_name = task_info["task_name"]
        task_tag = task_info["task_tag"]
        nosql_table = task_info["nosql_table"]
        latest = task_info["table_latest"]
        conn_latest = Coll.get_table_conn(latest)
        # 对需要的元素个数进行检查, 取传进来的id列表中第一个元素进行检查, 默认后续数据与第一个格式相同, 约定以字典形式进行传入
        bools, msg = self.check_article_meta(data_dicts_list[0])
        if bools:
            new_id_list = [data["lngid"] for data in data_dicts_list]

            data_latest = conn_latest.find({"_id": {"$in": new_id_list}}, {"rawid_mysql": 1})
            async for adata in data_latest:
                key = adata["_id"]
                rawid = adata["rawid_mysql"]
                _id = f"{task_name}_{task_tag}_{rawid}"
                self.a_dict[key] = _id
                self.a_list.append(_id)
            conn = Coll.get_table_conn(nosql_table)
            data_list = conn.find({"_id": {"$in": self.a_list}}, infos)
            async for data in data_list:
                local_dict = dict()
                last_data = data["last_data"]
                down_dict = last_data["down_dict"]
                for local_key, local_value in self.a_dict.items():
                    if local_value == data["_id"]:
                        local_dict[local_key] = down_dict
                        self.b_list.append(local_dict)
        else:
            if msg == "no_value":
                return False, "所传字段错误, 请检查传输字典键: [{'lngid':''}]"
            elif msg == "not_equal":
                return False, "所传字段数量错误, 查询所需格式: [{'lngid':''}]"

    async def run(self, get_type, task_info, infos, id_list):
        if get_type == "list":
            bools, msg = await self.set_task_list(task_info, infos, id_list)
            if not bools:
                return False, msg
        elif get_type == "issue":
            bools, msg = await self.set_task_issue(task_info, infos, id_list)
            if not bools:
                return False, msg
        elif get_type == "article":
            await self.set_task_article(task_info, infos, id_list)
        else:
            return False, "get_type错误,目前只有list, issue, article, 请检查"
        return True, self.b_list


class GetJournalHtmlDetailExport(object):
    def __init__(self):
        self.b_list = list()
        self.a_list = list()
        self.a_dict = dict()
        self.issue_dict = dict()

    def check_meta(self, rule, data):
        '''


        :param rule: 组成查询id的规则
        :param data: 单条数据
        :return:
        '''
        if "page_index" in rule:
            if len(rule) - 1 == len(data):
                for i in rule:
                    if i != "page_index":
                        meta_1 = data.get(i)
                        if meta_1 is None:
                            return False, "no_value"
                return True, ""
            else:
                return False, "not_equal"
        else:
            if len(rule) == len(data):
                for i in rule:
                    meta_1 = data.get(i)
                    if meta_1 is None:
                        return False, "no_value"
                return True, ""
            else:
                return False, "not_equal"

    def check_article_meta(self, data):
        '''
         针对文章级提取需求
        :param data:
        :return:
        '''
        if len(data) == 1:
            for k, v in data.items():
                if k == "lngid":
                    return True, ""
            return False, "no_value"
        else:
            return False, "not_equal"

    def check_page_index_mode(self, id_list):
        if "page_index" in id_list:
            return "on"
        else:
            return "off"

    async def set_task_all(self, task_info, infos, data_dicts_list):
        task_name = task_info["task_name"]
        task_tag = task_info["task_tag"]
        # redis里面的id_list
        id_list_parse = task_info["id_list"]
        nosql_table = task_info["nosql_table"]
        table_name = task_info["table_name"]
        force_index = task_info["force_index"]
        # latest = task_info["table_latest"]
        rule_list = eval(id_list_parse)
        judge = self.check_page_index_mode(rule_list)
        if judge == "on":
            bools, msg = self.check_meta(rule_list, data_dicts_list[0])
            if bools:
                new_id_list = list()
                for data_dict in data_dicts_list:
                    rawid_list = []
                    for name in rule_list:
                        for data_key, data_value in data_dict.items():
                            if name == data_key:
                                rawid_list.append(data_value)
                    _id = "_".join(rawid_list)
                    # 获取where的条件
                    sql_where_list = []
                    for k, v in data_dict.items():
                        value = f"{k}='{v}'"
                        sql_where_list.append(value)
                    where = " and ".join(sql_where_list)
                    # 这个列表用于将这个键下所有页码的id记录在内
                    self.a_dict[_id] = []
                    self.issue_dict[_id] = []
                    sql = f"select page from {table_name} where {where} and page_index in (0, 1)"
                    from main import app
                    pool = app.state.pool
                    data = await SQLHelper.fetchone(sql)
                    for i in range(1, int(data[0]) + 1):
                        one_data_list = [_id]
                        one_data_list.append(str(i))
                        one_page_id = "_".join(one_data_list)
                        # 这个列表用于等会统一查询
                        new_id_list.append(one_page_id)
                        self.a_dict[_id].append(one_page_id)
                conn = Coll.get_table_conn(nosql_table)
                data_list = conn.find({"_id": {"$in": new_id_list}}, infos)
                async for data in data_list:
                    local_dict = dict()
                    last_data = data["last_data"]
                    down_dict = last_data["down_dict"]
                    local_key = data["_id"]
                    local_dict[local_key] = down_dict
                    for key, value in self.a_dict.items():
                        if local_key in value:
                            self.issue_dict[key].append(local_dict)
                for k, v in self.issue_dict.items():
                    self.b_list.append({k: v})
                return True, ""
            else:
                if msg == "no_value":
                    return False, "所传字段错误, 请检查传输字典键: {}, task_name与task_tag与page_index不需要传入,其余需传入".format(id_list_parse)
                elif msg == "not_equal":
                    return False, "所传字段数量错误, 查询所需元素: {}, task_name与task_tag与page_index不需要传入,其余需传入".format(id_list_parse)

        elif judge == "off":
            # 对需要的元素个数进行检查, 取传进来的id列表中第一个元素进行检查, 默认后续数据与第一个格式相同, 约定以字典形式进行传入
            bools, msg = self.check_meta(rule_list, data_dicts_list[0])
            if bools:
                new_id_list = list()
                for data_dict in data_dicts_list:
                    one_data_list = [task_name, task_tag]
                    for list_meta in rule_list:
                        if list_meta not in ["task_name", "task_tag"]:
                            meta_data = BaseDicts.is_dict_exit_key(data_dict, list_meta)
                            one_data_list.append(meta_data)
                            data_id = "_".join(one_data_list)
                            new_id_list.append(data_id)
                conn = Coll.get_table_conn(nosql_table)
                data_list = conn.find({"_id": {"$in": new_id_list}}, infos)
                async for data in data_list:
                    local_dict = dict()
                    last_data = data["last_data"]
                    down_dict = last_data["down_dict"]
                    local_key = data["_id"]
                    local_dict[local_key] = down_dict
                    self.b_list.append(local_dict)
                return True, ""
            else:
                if msg == "no_value":
                    return False, "所传字段错误, 请检查传输字典键: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)
                elif msg == "not_equal":
                    return False, "所传字段数量错误, 查询所需元素: {}, task_name与task_tag不需要传入,其余需传入".format(id_list_parse)
        else:
            return False, ""

    async def set_task_latest(self, task_info, infos, data_dicts_list):
        task_name = task_info["task_name"]
        task_tag = task_info["task_tag"]
        nosql_table = task_info["nosql_table"]
        latest = task_info["table_latest"]
        conn_latest = Coll.get_table_conn(latest)
        # 对需要的元素个数进行检查, 取传进来的id列表中第一个元素进行检查, 默认后续数据与第一个格式相同, 约定以字典形式进行传入
        bools, msg = self.check_article_meta(data_dicts_list[0])
        if bools:
            new_id_list = [data["lngid"] for data in data_dicts_list]

            data_latest = conn_latest.find({"_id": {"$in": new_id_list}}, {"rawid_mysql": 1})
            async for adata in data_latest:
                key = adata["_id"]
                # 因出现提取老数据的情况, 老数据没有rawid_mysql导致提取错误, 因此跟苏鸿商量后, 更改为现在程序,错误数据跳过
                rawid = adata.get("rawid_mysql")
                if rawid is None:
                    continue
                _id = f"{task_name}_{task_tag}_{rawid}"
                self.a_dict[key] = _id
                self.a_list.append(_id)
            # conn = Coll.get_table_conn(nosql_table)
            # data_list = conn.find({"_id": {"$in": self.a_list}}, infos)
            # print(self.a_list)
            data = InputPlatformModel(
                data=SelectMongoModel(
                    field=infos,
                    table=nosql_table,
                    and_param={"_id": {"$in": self.a_list}},
                    limit=len(self.a_list)
                )).json()
            url = await URLDISTRIBUTED.get_select_url()
            rrq = RequestApiHelper.get_rrq()
            kwargs = {
                "rrq": rrq,
                "header": INSIDE_HEADERS,
                "url": url,
                "timeout": 30,
                "data": data,
                "moths": MRequest.POST,
                "middler_list": [rrq.status_code_middlerwares, rrq.is_none_html_middlerwares,
                                 rrq.is_null_html_middlerwares,
                                 rrq.msg_status_code_middlerwares]
            }
            bools, dicts = await RequestApiHelper.mongo_save(**kwargs)
            if bools:
                html = json.loads(rrq.html)
                data_list = html["data"]
                for data in data_list:
                    local_dict = dict()
                    last_data = data["last_data"]
                    down_dict = last_data.get("down_dict")
                    if down_dict is None:
                        continue
                    data_id = data.get("_id", "")
                    if data_id == "":
                        para = data.get("para", "")
                        id_list = para["id_list"]
                        ids_list = [para[name] for name in id_list]
                        data_id = "_".join(ids_list)
                    for local_key, local_value in self.a_dict.items():
                        if local_value == data_id:
                            local_dict[local_key] = down_dict
                            self.b_list.append(local_dict)
        else:
            if msg == "no_value":
                return False, "所传字段错误, 请检查传输字典键: [{'lngid':''}]"
            elif msg == "not_equal":
                return False, "所传字段数量错误, 查询所需格式: [{'lngid':''}]"

    async def run(self, get_type, task_info, infos, id_list):
        if get_type == "latest":
            await self.set_task_latest(task_info, infos, id_list)
        else:
            bools, msg = await self.set_task_all(task_info, infos, id_list)
            if not bools:
                return False, msg
            # return False, "get_type错误,目前只有list, issue, article, 请检查"
        return True, self.b_list
